Remote compilation of BigDFT
In this notebook we provide examples about how to compile the code on a remote machine with the RemoteManager Interoperability module of PyBigDFT.
We know that the bigdft suite requires, for the compilation, a buildrc file which exposes the configure lines of the suite packages. In addition, it is possible to pre-compile the list of the upstream dependencies packages which are related to the optional functionalities of the BigDFT suite. This is a practical advantage as it would enable a easier update of the bigdft suite when new version are released.
Let us now illustrate the remote compilation with the aquarius and odyssey machines of Tokyo university.
To start doing that, we need a dictionary-based specification of the installation of the code. We use a yaml configuration string for practical reasons.
[10]:
wisteria_spec="""
submitter: pjsub
# this is for both odyssey and
template: |
#!/bin/sh
# #cores_per_node:default=48# #feature of the machine
#------ pjsub option --------#
#PJM -L rscgrp=#queue:default=regular-a#
#PJM -L node=#nodes:default={(mpi*omp)/cores_per_node}#
#PJM -L gpu=#gpu#
#PJM --mpi proc=#mpi:optional=False#
#PJM --omp thread=#omp:optional=False#
#PJM -L elapse=#time:format=time:optional=False#
#PJM -g #group:default=jh210022a#
#PJM -j
module #module_preload#
module load #modules#
export PREFIX=#prefix:default=/work/jh210022a/q25020/build-odyssey/suite#
export BIGDFT_MPIRUN="mpiexec #mpiexec_extra:default=-machinefile $PJM_O_NODEINF -n $PJM_MPI_PROC -npernode {min(cores_per_node/omp,mpi)}#"
export FUTILE_PROFILING_DEPTH=0
export FUTILE_DEBUG_MODE=#futile_debug:default=0#
source $PREFIX/bin/bigdftvars.sh
export GENESIS_ROOT=#upstream_prefixes#/bin
export GENESIS_MPIRUN=$BIGDFT_MPIRUN
#------- Program execution -------#
frontend_template: |
#!/bin/bash
module #module_preload#
module load #modules#
export PREFIX=#prefix:default=/work/jh210022a/q25020/build-odyssey/suite#
export FUTILE_PROFILING_DEPTH=0
export FUTILE_DEBUG_MODE=#futile_debug:default=0#
source $PREFIX/bin/bigdftvars.sh
export GENESIS_ROOT=#upstream_prefixes#/bin
export GENESIS_MPIRUN=$BIGDFT_MPIRUN
#command:optional=False#
odyssey_gnu_nompi:
fc: gfortran
fcflags: -fPIC
cc: gcc
cflags: -fPIC
cxx: g++
modules: fj
python_interpreter: python
sourcedir: /home/q25020/bigdft-suite
tarballdir: /home/q25020/bigdft-upstream-tarballs
checkoutroot: /work/jh210022a/q25020/build-odyssey/checkout
builddir: /work/jh210022a/q25020/build-odyssey/builddir
upstream_prefixes: /work/jh210022a/q25020/build-odyssey/upstream
prefix: /work/jh210022a/q25020/build-odyssey/suite
odyssey:
fc: mpifrtpx
fcflags: -SSL2BLAMP -Kfast,openmp,noautoobjstack -fPIC
cc: mpifccpx
cflags: -g -std=gnu99 -O2 -fPIC
cxx: mpiFCCpx
linalg: -fjlapackex -SCALAPACK
configure_line: F77=frtpx FFLAGS="-Kfast -fPIC" --build=x86_64-redhat-linux --host=sparc-fujitsu-linux LIBS="-SSL2BLAMP -Kfast,openmp -Nlibomp" --without-archives
module_preload: purge
modules: fj
python_interpreter: python
sourcedir: /home/q25020/bigdft-suite
mpiexec_extra: ""
group: jh210022o
queue: regular-o
cmakeargs_update: >
{'ntpoly': ' -DFORTRAN_ONLY=Yes -DCMAKE_Fortran_FLAGS_RELEASE="-SSL2BLAMP -Kfast,openmp" -DCMAKE_Fortran_COMPILER="mpifrtpx" -DCMAKE_C_COMPILER="mpifccpx" -DCMAKE_CXX_COMPILER="mpiFCCpx" -DCMAKE_Fortran_MODDIR_FLAG="-M"'}
tarballdir: /home/q25020/bigdft-upstream-tarballs
checkoutroot: /work/jh210022a/q25020/build-odyssey/checkout
builddir: /work/jh210022a/q25020/build-odyssey/builddir
upstream_prefixes: /work/jh210022a/q25020/build-odyssey/upstream
prefix: /work/jh210022a/q25020/build-odyssey/suite
aquarius:
fc: mpif90
fcflags: -O2 -g -fPIC -fopenmp
cc: mpicc
cflags: -O2 -g -fPIC
ompflags: -fopenmp
cxx: mpicxx
linalg: -L${MKLROOT}/lib/intel64 -Wl,--no-as-needed -lmkl_gf_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl
configure_line: --enable-cuda-gpu --enable-opencl "NVCC_FLAGS=--compiler-options -fPIC" --enable-single --enable-gpu --with-cuda=$CUDA_PATH LAPACK_LIBS="-L${MKLROOT}/lib/intel64 -Wl,--no-as-needed -lmkl_gf_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl" --enable-dynamic-libraries LIBS=-lstdc++
#gpu_line:
build_conditions: --conditions=-vdw
module_preload: purge
modules: aquarius python/3.9.18 ompi mkl cuda
python_interpreter: python
sourcedir: /home/q25020/bigdft-suite
tarballdir: /home/q25020/bigdft-upstream-tarballs
checkoutroot: /work/jh210022a/q25020/build-aquarius/checkout
builddir: /work/jh210022a/q25020/build-aquarius/builddir
upstream_prefixes: /work/jh210022a/q25020/build-aquarius/upstream
prefix: /work/jh210022a/q25020/build-aquarius/suite
"""
Let us first verify if the connection with the remote computer can be instantiated.
[11]:
from yaml import load, Loader
from remotemanager import BaseComputer
wisteria_dict = load(wisteria_spec,Loader=Loader)
[12]:
wisteria = BaseComputer(host='wisteria.cc.u-tokyo.ac.jp', user='q25020', **wisteria_dict)
#wisteria.test_connection()
With these configuration options, the buildrc can be created for the desired platform
[13]:
key='odyssey'
rec=wisteria_dict[key]
[14]:
from BigDFT.Interop import RemoteManagerInterop as RM
buildrc=BaseComputer(template=RM.buildrc_template)
rcfile='wisteria-'+key+'.rc'
with open(rcfile,'w') as ofile:
ofile.write(buildrc.script(targets=['upstream-suite'],conditions=['ase'],**rec))
rcfile_gnu='wisteria-gnu.rc'
with open(rcfile_gnu,'w') as ofile:
ofile.write(buildrc.script(targets=['client-bio-baseplugins'], conditions=['vdw', 'spg'],**wisteria_dict['odyssey_gnu_nompi']))
Then we can start to compile the code. We will split the compilation in two steps, first the compilation of the upstream packages, then the bigdft-suite.
[15]:
from os import environ, path
#rcfile=path.join(environ['BIGDFT_SUITE_SOURCES'], 'rcfiles', 'fugaku_cross.rc') #alternative choice
cp=RM.code_compiler(extra_files_send=[rcfile],
host='wisteria.cc.u-tokyo.ac.jp', user='q25020',
local_dir='compile',
rcfile=path.basename(rcfile),
asynchronous=False,**rec)
[16]:
cp.append_run(checkoutroot=rec['builddir'],action='build', upstream_prefixes='',
prefix=rec['upstream_prefixes'])
cp.append_run(checkoutroot=rec['builddir'],action='build dftd3', upstream_prefixes='',
prefix=rec['upstream_prefixes'],rcfile=rcfile_gnu,extra_file_send=rcfile_gnu)
cp.append_run(action='build spred',build_conditions="--conditions=+no_upstream")
appended run runner-0
appended run runner-1
appended run runner-2
[17]:
cp.run()
Running Dataset
assessing run for runner compile-c48c306e-runner-0... running
assessing run for runner compile-c48c306e-runner-1... running
assessing run for runner compile-c48c306e-runner-2... running
Transferring 6 Files in 2 Transfers... Done
The same procedure can be performed for the aquarius platform
[18]:
key='aquarius'
rec=wisteria_dict[key]
[19]:
rcfile='wisteria-'+key+'.rc'
with open(rcfile,'w') as ofile:
ofile.write(buildrc.script(targets=['upstream-suite'],conditions=['ase'],**rec))
[20]:
!rm dataset-compile-c48c306e.yaml
[21]:
cp=RM.code_compiler(extra_files_send=[rcfile],
host='wisteria.cc.u-tokyo.ac.jp', user='q25020',
local_dir='compile',
rcfile=path.basename(rcfile),
asynchronous=False,**rec)
cp.append_run(checkoutroot=rec['builddir'],action='buildone genesis', upstream_prefixes='',
prefix=rec['upstream_prefixes'])
cp.append_run(checkoutroot=rec['builddir'],action='build', upstream_prefixes='',
prefix=rec['upstream_prefixes'])
cp.append_run(action='build spred',build_conditions="--conditions=+no_upstream")
appended run runner-0
appended run runner-1
appended run runner-2
[22]:
cp.run()
Running Dataset
assessing run for runner compile-c48c306e-runner-0... running
assessing run for runner compile-c48c306e-runner-1... running
assessing run for runner compile-c48c306e-runner-2... running
Transferring 6 Files in 2 Transfers... Done
Update the suite remotely before compiling
Sometimes, (especially when the frontend has no internet access) it may be useful to update remotely the code before starting the compilation procedure. We illustrate this approach with the irene platform. Of course, a git repository should be present in the remote directory.
[5]:
from remotemanager import BaseComputer
irene=BaseComputer( host='irene', passfile='/tmp/irene')
[6]:
irene.test_connection()
Checking for entry point... Success (Error: failed to set /sys/fs/cgroup/cpuset/regular_users/genovesl/cpuset.cpus: Permission denied
Error: failed to set /sys/fs/cgroup/cpuset/regular_users/genovesl/cpuset.mems: Permission denied
/ccc/cont003/dsku/blanchet/home/user/drf/genovesl)
Checking file creation in home... True
Checking file creation in /tmp... True
Checking file creation in /scratch... False
Testing remotemanager.transport.rsync:
send... Transferring 3 Files... Done
True
pull... Transferring 1 File... Done
True
Testing remotemanager.transport.scp:
send... Transferring 3 Files... (1/3) communication attempt failed after 5s... waiting 5s and trying again
Done
(1/3) communication attempt failed after 5s... waiting 5s and trying again
False
pull... Transferring 1 File... Done
True
Cleaning up... Done
Done! Made 17 calls, taking 59.62s
Approximate latency, 3.51s
Tests passed successfully
[7]:
from BigDFT.Interop import RemoteManagerInterop as RM
[9]:
RM.code_updater(remote_sourcedir='/ccc/work/cont003/drf/genovesl/1.9.5',
host='irene', local_dir='compile', passfile='/tmp/irene')
appended run runner-0
Running Dataset
assessing run for runner push_to_remote-e61e2906-runner-0... force running
Transferring 3 Files... Done
Warning! Found 1 error(s), also check the `errors` property!
Including git+ssh://irene/ccc/work/cont003/drf/genovesl/1.9.5 as irene
Delete remote target if non empty
Pushing external repo
Remote "irene" does not support the LFS locking API. Consider disabling it with:
$ git config lfs.https://irene/ccc/work/cont003/drf/genovesl/1.9.5.git/info/lfs.locksverify false
Pushing ended
appended run runner-0
Running Dataset
assessing run for runner update_remote-c610c176-runner-0... force running
Transferring 3 Files... Done
Warning! Found 1 error(s), also check the `errors` property!
Pulling external repo into build
Your branch is ahead of 'irene/tmp_update' by 209 commits.
(use "git push" to publish your local commits)
Your branch is ahead of 'irene/tmp_update' by 209 commits.
(use "git push" to publish your local commits)
Updating 2a206f222..79e15ef74
Fast-forward
.gitlab-ci.yml | 10 +-
PyBigDFT/BigDFT/BioQM.py | 8 +-
PyBigDFT/BigDFT/Interop/ASEInterop.py | 16 +-
PyBigDFT/BigDFT/Interop/DNAviewerInterop.py | 13 +-
PyBigDFT/BigDFT/Interop/RemoteManagerInterop.py | 328 +-
PyBigDFT/BigDFT/Interop/XTBInterop.py | 15 +-
PyBigDFT/pyproject.toml | 2 +-
PyBigDFT/setup.py | 2 +-
bigdft/src/cluster.f90 | 1 +
bigdft/src/helpers/Makefile.am | 3 +
bigdft/src/helpers/bigdft_environ.f90 | 27 +
bigdft/src/init.f90 | 3 +-
bigdft/src/linear/copyTypes.f90 | 6 +-
bigdft/src/linear/initAndUtils.f90 | 2 +
bigdft/src/linear/linearScaling.f90 | 3 +
bigdft/src/linear/transferintegrals.f90 | 2 +-
bigdft/src/modules/Makefile.am | 5 +-
bigdft/src/modules/bigdft_matrices.f90 | 11 +-
bigdft/src/modules/bigdft_run.f90 | 2 +
bigdft/src/modules/chess_utils.f90 | 253 +
bigdft/src/modules/dpbox.f90 | 7 -
bigdft/src/modules/forces_linear.f90 | 1 +
bigdft/src/modules/get_basis.f90 | 1 +
bigdft/src/modules/get_kernel.f90 | 158 +-
bigdft/src/modules/io.f90 | 3 +
bigdft/src/modules/multipole.f90 | 12 +-
bigdft/src/modules/orthonormalization.f90 | 11 +-
bigdft/src/modules/postprocessing_linear.f90 | 1 +
bigdft/src/modules/rhopotential.f90 | 5 +-
bigdft/src/modules/transposed_operations.f90 | 4 +-
bigdft/src/modules/types.f90 | 170 +-
bigdft/src/modules/unitary_tests.f90 | 15 +-
bigdft/src/restart.f90 | 1 +
bigdft/tests/DFT/linear/Makefile.am | 10 +
.../TCDFT_acetone_t300_000001_PBE_SF_S0.out.ref | 1 +
...CDFT_acetone_t300_000001_PBE_SF_S0.out.ref.yaml | 1893 ++++++
...ne_t300_000001_PBE_SF_S1_pure_HOMO_LUMO.out.ref | 2 +
...00_000001_PBE_SF_S1_pure_HOMO_LUMO.out.ref.yaml | 2684 ++++++++
...etone_t300_000001_PBE_SF_sp_spd_12.0_S0.out.ref | 2 +
..._t300_000001_PBE_SF_sp_spd_12.0_S0.out.ref.yaml | 2017 ++++++
...e_t300_000001_PBE_SF_sp_spd_12.0_linear.out.ref | 2 +
...0_000001_PBE_SF_sp_spd_12.0_linear.out.ref.yaml | 3502 ++++++++++
..._t300_000001_PBE_SF_sp_spd_12.0_virtual.out.ref | 2 +
..._000001_PBE_SF_sp_spd_12.0_virtual.out.ref.yaml | 7089 ++++++++++++++++++++
.../tests/DFT/linear/TCDFT/acetone_t300_000001.xyz | 12 +
.../TCDFT/acetone_t300_000001_PBE_SF_S0.yaml | 688 ++
...etone_t300_000001_PBE_SF_S1_pure_HOMO_LUMO.yaml | 700 ++
.../acetone_t300_000001_PBE_SF_sp_spd_12.0_S0.yaml | 688 ++
...tone_t300_000001_PBE_SF_sp_spd_12.0_linear.yaml | 677 ++
...one_t300_000001_PBE_SF_sp_spd_12.0_virtual.yaml | 690 ++
bigdft/tests/tols-BigDFT.yaml | 109 +-
chess/sparsematrix/sparsematrix.f90 | 31 -
chess/sparsematrix/sparsematrix_types.f90 | 5 -
futile/flib/f_environment.f90 | 41 +-
futile/tests/fldiff_yaml.py.in | 10 +-
modulesets/bio.modules | 2 +-
modulesets/suite.modules | 2 +-
57 files changed, 21649 insertions(+), 311 deletions(-)
create mode 100644 bigdft/src/helpers/bigdft_environ.f90
create mode 100644 bigdft/src/modules/chess_utils.f90
create mode 100644 bigdft/tests/DFT/linear/TCDFT/TCDFT_acetone_t300_000001_PBE_SF_S0.out.ref
create mode 100644 bigdft/tests/DFT/linear/TCDFT/TCDFT_acetone_t300_000001_PBE_SF_S0.out.ref.yaml
create mode 100644 bigdft/tests/DFT/linear/TCDFT/TCDFT_acetone_t300_000001_PBE_SF_S1_pure_HOMO_LUMO.out.ref
create mode 100644 bigdft/tests/DFT/linear/TCDFT/TCDFT_acetone_t300_000001_PBE_SF_S1_pure_HOMO_LUMO.out.ref.yaml
create mode 100644 bigdft/tests/DFT/linear/TCDFT/TCDFT_acetone_t300_000001_PBE_SF_sp_spd_12.0_S0.out.ref
create mode 100644 bigdft/tests/DFT/linear/TCDFT/TCDFT_acetone_t300_000001_PBE_SF_sp_spd_12.0_S0.out.ref.yaml
create mode 100644 bigdft/tests/DFT/linear/TCDFT/TCDFT_acetone_t300_000001_PBE_SF_sp_spd_12.0_linear.out.ref
create mode 100644 bigdft/tests/DFT/linear/TCDFT/TCDFT_acetone_t300_000001_PBE_SF_sp_spd_12.0_linear.out.ref.yaml
create mode 100644 bigdft/tests/DFT/linear/TCDFT/TCDFT_acetone_t300_000001_PBE_SF_sp_spd_12.0_virtual.out.ref
create mode 100644 bigdft/tests/DFT/linear/TCDFT/TCDFT_acetone_t300_000001_PBE_SF_sp_spd_12.0_virtual.out.ref.yaml
create mode 100644 bigdft/tests/DFT/linear/TCDFT/acetone_t300_000001.xyz
create mode 100644 bigdft/tests/DFT/linear/TCDFT/acetone_t300_000001_PBE_SF_S0.yaml
create mode 100644 bigdft/tests/DFT/linear/TCDFT/acetone_t300_000001_PBE_SF_S1_pure_HOMO_LUMO.yaml
create mode 100644 bigdft/tests/DFT/linear/TCDFT/acetone_t300_000001_PBE_SF_sp_spd_12.0_S0.yaml
create mode 100644 bigdft/tests/DFT/linear/TCDFT/acetone_t300_000001_PBE_SF_sp_spd_12.0_linear.yaml
create mode 100644 bigdft/tests/DFT/linear/TCDFT/acetone_t300_000001_PBE_SF_sp_spd_12.0_virtual.yaml
Merging finished
Now we can recompile the code in by only updating the suite, with the different compilation flavours
[10]:
import yaml
templates=yaml.load(RM.computers_database,Loader=yaml.Loader)
[18]:
recip={k:templates['irene'][k] for k in ['gnu_mpi', 'intel_mpi', 'intel_intelmpi', 'intel_oneapi_mpi']}
[27]:
from os import remove
buildrc=BaseComputer(template=RM.buildrc_template)
for irec,(key,rec) in enumerate(recip.items()):
rcfile='irene-'+key+'.rc'
with open(rcfile,'w') as ofile:
ofile.write(buildrc.script(targets=['upstream-suite', 'client-bio-baseplugins'],
conditions=['ase', 'vdw', 'spg'],**rec))
cp=RM.code_compiler(extra_files_send=[rcfile],
host='irene', local_dir='compile', passfile='/tmp/irene',
rcfile=rcfile,
asynchronous=False,force=True,**rec)
# cp.append_run(checkoutroot=rec['builddir'],action='build', upstream_prefixes='',
# prefix=rec['upstream_prefixes'])
cp.append_run(action='build -f bigdft',build_conditions="--conditions=+no_upstream")
cp.run()
remove(cp.database.path)
appended run runner-0
Running Dataset
assessing run for runner compile-c48c306e-runner-0... running
Transferring 4 Files in 2 Transfers... Done
appended run runner-0
Running Dataset
assessing run for runner compile-c48c306e-runner-0... running
Transferring 4 Files in 2 Transfers... Done
appended run runner-0
Running Dataset
assessing run for runner compile-c48c306e-runner-0... running
Transferring 4 Files in 2 Transfers... Done
[ ]: